package com.songmy.spider.run.my;

import java.util.Iterator;
import java.util.Map;
import java.util.Set;

import org.apache.http.HttpHost;
import org.apache.log4j.Logger;

import com.alibaba.fastjson.JSONObject;
import com.songmy.spider.run.vo.Field;
import com.songmy.spider.run.vo.MySelectorType;
import com.songmy.spider.run.vo.SpiderRule;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;

/***
 * 一般接口
 * @author Marc
 * @version 1.0
 */
public class UsuallySpiderProcessor implements PageProcessor {

	private SpiderRule rule = null ; 
	
	private Site site = Site.me().setRetryTimes(3).setSleepTime(100);
	
	private Logger log = Logger.getLogger(UsuallySpiderProcessor.class);
	
	public final static String _VALUE = "_VALUE";
	
	public UsuallySpiderProcessor(SpiderRule rule ) {
		this.rule = rule ;
		init();
		
	}
	

	/***
	 * 初始化过程,主要初始化
	 * @user Marc
	 * @createtime 2016年9月22日 下午5:48:10
	 */
	private void init() {
		createSite();
	}


	private void createSite() {
		int retryTimes = this.rule.getRetryTimes();
		int sleepTimes = this.rule.getSleepTime();
		Map<String, String> map = this.rule.getHead();
		site = Site.me().setRetryTimes(retryTimes).setSleepTime(sleepTimes);
		site.setDomain(this.rule.getDomains());
		if(map != null && map.size() > 0 ){
			Iterator<String>  it = map.keySet().iterator();
			while (it.hasNext()) {
				String key = it.next();
				String value = map.get(key);
				site.addHeader(key, value);
			}
		}
		
		if(this.rule.isUseProxy()){
			String hostname = this.rule.getHostname();
			int port = this.rule.getPort();
			String scheme_name = this.rule.getScheme_name();
			
			HttpHost proxy = new HttpHost(hostname, port, scheme_name);
			site.setHttpProxy(proxy);
		}
		
	}

	/***
	 * 规则,设置一个简单的爬虫规则
	 * @user Marc
	 * @createtime 2016年9月22日 下午1:00:03
	 */
	public void setSpiderRule(SpiderRule rule){
		this.rule = rule ;
	}
	
	public void process(Page page) {
		//添加采集地址
			
			try {
				for (int i = 0; i < this.rule.getAddUrlReg().size(); i++) {
					 page.addTargetRequests(page.getHtml().links().regex(this.rule.getAddUrlReg().get(i)).all());
				}
				for (int i = 0; i < this.rule.getContentUrlReg().size(); i++) {
					 page.addTargetRequests(page.getHtml().links().regex(this.rule.getContentUrlReg().get(i)).all());
				}
				//做字段解析等操作
				JSONObject o = new JSONObject();
				o.put("url", page.getRequest().getUrl());
				for (int i = 0; i < this.rule.getFields().size(); i++) {
					Field  f = this.rule.getFields().get(i);
					
					//设置的选择器
					String value = null ;
					if(f.getSelectorType() == MySelectorType.XPATH){
						value = page.getHtml().xpath(f.getSelectorRule()).toString();
					}else if(f.getSelectorType() == MySelectorType.JSON){
						value = page.getHtml().jsonPath(f.getSelectorRule()).toString();
					}else if(f.getSelectorType() == MySelectorType.REGEX){
						value = page.getHtml().regex(f.getSelectorRule()).toString();
					}else if(f.getSelectorType() == MySelectorType.ID){
						value = page.getHtml().$(f.getSelectorRule()).toString();
					}else if(f.getSelectorType() == MySelectorType.CSS){
						value = page.getHtml().css(f.getSelectorRule()).toString();
					}else{
						log.error("未找到解析器:"+f.getFieldCode()); 
					}
					
					
					
//					必输情况下,若value值为空 ;并且没有子字段,则抛弃当前值
					if(f.isRequired() && (f.getChildFied() == null || f.getChildFied().size() ==0 ) && value== null ){
						page.setSkip(true);
						continue;
					}
					/***
					 * value 替换
					 */
					Map<String, Object> map = f.getReplace();
					if(value!=null && map !=null && map.size() > 0){
						Set<String> setKey = map.keySet();
						Iterator<String> itKey = setKey.iterator();
						for (String key : setKey) {
							value = value.replace(key, map.get(key).toString());
						}
					}
					o.put(f.getFieldCode(), value);
				}
				page.putField(_VALUE, o);
			} catch (Exception e) {
				e.printStackTrace();
			}
		
		
		
		
	}

	public Site getSite() {

		return site;
	}

}
